/**
 * 
 */
package com.istart.webmagic.guba;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.JsonPathSelector;

/**
 * 文字表 的评论数和阅读数
 */
public class GuBaAuthorPageProcessor implements PageProcessor {
	//http://guba.eastmoney.com/default_1.html
    public static final String URL_LIST = "http://guba\\.eastmoney\\.com/default_\\d+\\.html";
//    http://guba.eastmoney.com/news,gssz,257496576.html
//    http://guba.eastmoney.com/news,zsgjgdaxi,257494345.html
    public static final String URL_POST = "http://guba\\.eastmoney\\.com/news,\\w+,\\d+\\.html";

    public static final String URL_NEXT = "default_\\d+\\.html";
    private Site site = Site
            .me()
            .setDomain("guba.eastmoney.com/")
            .setSleepTime(3000)
            .setUserAgent(
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
    @Override
    public void process(Page page) {
    	//这里我们使用JSONPATH这种选择语言来选择数据
//        List<String> ids = new JsonPathSelector("$.data[*]._id").selectList(page.getRawText());
//        if (CollectionUtils.isNotEmpty(ids)) {
//            for (String id : ids) {
//                page.addTargetRequest("http://angularjs.cn/api/article/"+id);
//            }
//        }
        
        String xx = page.getRawText().replace("jQuery18304588693571276963_1455807681608(", "").replace(")", "");
        page.putField("user_id", new JsonPathSelector("$.user_id").select(xx));
        page.putField("user_nickname", new JsonPathSelector("$.user_nickname").select(xx));
        page.putField("user_age", new JsonPathSelector("$.user_age").select(xx));
        page.putField("user_influ_level", new JsonPathSelector("$.user_influ_level").select(xx));
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new GuBaAuthorPageProcessor())
//        .addUrl("http://guba.eastmoney.com/news,gssz,258300805.html")
        	.addUrl("http://iguba.eastmoney.com/interf/user.aspx?callback=jQuery18304588693571276963_1455807681608&action=floatinfo&uid=1302113021957504&_=1455808002928")
        	.addPipeline(new JsonFilePipeline("G:/webmagic/"))
        	.thread(5)
            .run();
    }
}